package com.tongna.ucrawler;


import com.tongna.ucrawler.domain.SiteUrl;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import jodd.http.HttpBrowser;
import jodd.http.HttpRequest;
import jodd.http.HttpResponse;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class DemoCrawler implements Crawler {

  private PageVisitor pageVisitor;

  private Logger logger = LoggerManger.logger("demo");

  private String charset = "utf8";

  private List<String> hosts = new ArrayList<>();

  /**
   * 最大访问深度
   */
  private int maxDepth;

  private Long beginTime;

  private List<String> suffixs = new ArrayList<>();

  public void addSuffix(String suffix) {
    this.suffixs.add(suffix);
  }

  /**
   * 最大访问次数
   */
  private int maxSize;

  public DemoCrawler() {
    this.maxDepth = 10;
    this.maxSize = 10000;
    this.beginTime=System.currentTimeMillis();
  }


  public int getMaxDepth() {
    return maxDepth;
  }

  public void setMaxDepth(int maxDepth) {
    this.maxDepth = maxDepth;
  }

  public int getMaxSize() {
    return maxSize;
  }

  public void setMaxSize(int maxSize) {
    this.maxSize = maxSize;
  }

  private List<SiteUrl> urls = new ArrayList<>();

  public String getCharset() {
    return charset;
  }

  public void setCharset(String charset) {
    this.charset = charset;
  }

  public PageVisitor getPageVisitor() {
    return pageVisitor;
  }

  public void setPageVisitor(PageVisitor pageVisitor) {
    this.pageVisitor = pageVisitor;
  }

  /**
   * 网页种子
   */
  private List<String> seeds = new ArrayList<>();

  public void addSeed(String seed) {
    seeds.add(seed);
  }

  public void start() {

    new Thread(new Runnable() {
      @Override
      public void run() {
        work();
      }
    }).start();
  }

  public void work() {

    for (String url : seeds) {
      SiteUrl siteurl = new SiteUrl();
      siteurl.setLevel(1);
      siteurl.setUrl(url);
      urls.add(siteurl);
      HttpRequest request = HttpRequest.get(url);
      this.hosts.add(request.host());
    }
    List<SiteUrl> visitUrls=new ArrayList<>();
    visitUrls.addAll(urls);
    handle(visitUrls);
    Long time=System.currentTimeMillis()-this.beginTime;
    time=time/1000;
    System.out.println("************end*************");
    System.out.println("************耗时*************:"+time+"秒");

  }

  public void handle(List<SiteUrl> visitUrls) {
    for (SiteUrl seed : visitUrls) {
      logger.info("visit:" + seed);
      String html = "";
      try {
        html = handlePage(seed);
      } catch (Exception e) {
        logger.info("错误:" + e.getMessage());
      }
      if (pageVisitor != null) {
        pageVisitor.visit(html);
      }
      if (urls.size() >= maxSize) {
        break;
      }
      if (seed.getLevel() >= maxDepth) {
        break;
      }
      List<SiteUrl> urlList = extractUrls(seed, html);
      if (urlList.size() > 0) {
        handle(urlList);
      }
    }
  }

  private String handlePage(SiteUrl seed) {
    HttpBrowser browser = getHttpBrowser();
    HttpRequest request = HttpRequest.get(seed.getUrl());
    request.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0");
    request.timeout(5000);
    request.connectionTimeout(5000);

    HttpResponse response = request.send();
    response.charset(this.charset);
    return response.bodyText();
  }

  private HttpBrowser getHttpBrowser() {
    HttpBrowser browser = new HttpBrowser();
    browser.setKeepAlive(false);
    browser.setDefaultHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0");
    return browser;
  }


  private List<SiteUrl> extractUrls(SiteUrl seed, String html) {
    List<SiteUrl> urlList = new ArrayList<>();
    Document document = Jsoup.parse(html);
    Elements es = document.select("a");
    for (Element e : es) {
      String href = e.attr("href");
      if (href == null) {
        continue;
      }
      if (!href.startsWith("http")) {
        HttpRequest request = HttpRequest.get(seed.getUrl());
        String hosturl = request.hostUrl();
        if (hosturl.lastIndexOf("/") != 0) {
          if (href.startsWith("/")) {
            href = request.hostUrl() + href;
          } else {
            href = request.hostUrl() + "/" + href;
          }
        } else {
          href = request.hostUrl() + href;
        }
      }
      SiteUrl url = new SiteUrl();
      url.setParent(seed);
      url.setUrl(href);
      if (haveAddUrl(url)) {
        urls.add(url);
        urlList.add(url);
      }
    }
    return urlList;
  }

  private boolean haveAddUrl(SiteUrl url) {
    boolean result = true;
    for (String suffix : suffixs) {
      if (url.getUrl().endsWith(suffix)) {
        return false;
      }
    }
    if (this.urls.contains(url)) {
      return false;
    }
    for (String host : hosts) {
      if (!url.getUrl().contains(host)){
        return false;
      }
    }


    return result;
  }
}
